{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple linear regression\n",
    "Solution using MLP and Linear Algebra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_4 (Dense)              (None, 8)                 16        \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 1)                 9         \n",
      "=================================================================\n",
      "Total params: 25\n",
      "Trainable params: 25\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/100\n",
      "10/10 [==============================] - 0s 8ms/sample - loss: 9.5318\n",
      "Epoch 2/100\n",
      "10/10 [==============================] - 0s 647us/sample - loss: 9.3012\n",
      "Epoch 3/100\n",
      "10/10 [==============================] - 0s 591us/sample - loss: 9.1035\n",
      "Epoch 4/100\n",
      "10/10 [==============================] - 0s 742us/sample - loss: 8.9039\n",
      "Epoch 5/100\n",
      "10/10 [==============================] - 0s 581us/sample - loss: 8.6748\n",
      "Epoch 6/100\n",
      "10/10 [==============================] - 0s 561us/sample - loss: 8.4875\n",
      "Epoch 7/100\n",
      "10/10 [==============================] - 0s 835us/sample - loss: 8.3100\n",
      "Epoch 8/100\n",
      "10/10 [==============================] - 0s 574us/sample - loss: 8.1167\n",
      "Epoch 9/100\n",
      "10/10 [==============================] - 0s 731us/sample - loss: 7.9459\n",
      "Epoch 10/100\n",
      "10/10 [==============================] - 0s 551us/sample - loss: 7.7874\n",
      "Epoch 11/100\n",
      "10/10 [==============================] - 0s 721us/sample - loss: 7.6144\n",
      "Epoch 12/100\n",
      "10/10 [==============================] - 0s 627us/sample - loss: 7.4491\n",
      "Epoch 13/100\n",
      "10/10 [==============================] - 0s 685us/sample - loss: 7.2856\n",
      "Epoch 14/100\n",
      "10/10 [==============================] - 0s 741us/sample - loss: 7.1208\n",
      "Epoch 15/100\n",
      "10/10 [==============================] - 0s 806us/sample - loss: 6.9644\n",
      "Epoch 16/100\n",
      "10/10 [==============================] - 0s 908us/sample - loss: 6.8253\n",
      "Epoch 17/100\n",
      "10/10 [==============================] - 0s 843us/sample - loss: 6.6876\n",
      "Epoch 18/100\n",
      "10/10 [==============================] - 0s 914us/sample - loss: 6.5457\n",
      "Epoch 19/100\n",
      "10/10 [==============================] - 0s 836us/sample - loss: 6.4058\n",
      "Epoch 20/100\n",
      "10/10 [==============================] - 0s 808us/sample - loss: 6.2617\n",
      "Epoch 21/100\n",
      "10/10 [==============================] - 0s 564us/sample - loss: 6.1252\n",
      "Epoch 22/100\n",
      "10/10 [==============================] - 0s 758us/sample - loss: 6.0038\n",
      "Epoch 23/100\n",
      "10/10 [==============================] - 0s 659us/sample - loss: 5.8782\n",
      "Epoch 24/100\n",
      "10/10 [==============================] - 0s 672us/sample - loss: 5.7208\n",
      "Epoch 25/100\n",
      "10/10 [==============================] - 0s 591us/sample - loss: 5.5900\n",
      "Epoch 26/100\n",
      "10/10 [==============================] - 0s 802us/sample - loss: 5.4749\n",
      "Epoch 27/100\n",
      "10/10 [==============================] - 0s 692us/sample - loss: 5.3588\n",
      "Epoch 28/100\n",
      "10/10 [==============================] - 0s 713us/sample - loss: 5.2277\n",
      "Epoch 29/100\n",
      "10/10 [==============================] - 0s 650us/sample - loss: 5.1127\n",
      "Epoch 30/100\n",
      "10/10 [==============================] - 0s 640us/sample - loss: 4.9844\n",
      "Epoch 31/100\n",
      "10/10 [==============================] - 0s 648us/sample - loss: 4.8724\n",
      "Epoch 32/100\n",
      "10/10 [==============================] - 0s 643us/sample - loss: 4.7594\n",
      "Epoch 33/100\n",
      "10/10 [==============================] - 0s 629us/sample - loss: 4.6411\n",
      "Epoch 34/100\n",
      "10/10 [==============================] - 0s 801us/sample - loss: 4.5377\n",
      "Epoch 35/100\n",
      "10/10 [==============================] - 0s 654us/sample - loss: 4.4286\n",
      "Epoch 36/100\n",
      "10/10 [==============================] - 0s 751us/sample - loss: 4.3177\n",
      "Epoch 37/100\n",
      "10/10 [==============================] - 0s 621us/sample - loss: 4.2238\n",
      "Epoch 38/100\n",
      "10/10 [==============================] - 0s 533us/sample - loss: 4.1333\n",
      "Epoch 39/100\n",
      "10/10 [==============================] - 0s 618us/sample - loss: 4.0302\n",
      "Epoch 40/100\n",
      "10/10 [==============================] - 0s 741us/sample - loss: 3.9403\n",
      "Epoch 41/100\n",
      "10/10 [==============================] - 0s 547us/sample - loss: 3.8522\n",
      "Epoch 42/100\n",
      "10/10 [==============================] - 0s 633us/sample - loss: 3.7623\n",
      "Epoch 43/100\n",
      "10/10 [==============================] - 0s 790us/sample - loss: 3.6706\n",
      "Epoch 44/100\n",
      "10/10 [==============================] - 0s 761us/sample - loss: 3.5948\n",
      "Epoch 45/100\n",
      "10/10 [==============================] - 0s 483us/sample - loss: 3.5092\n",
      "Epoch 46/100\n",
      "10/10 [==============================] - 0s 715us/sample - loss: 3.4238\n",
      "Epoch 47/100\n",
      "10/10 [==============================] - 0s 536us/sample - loss: 3.3391\n",
      "Epoch 48/100\n",
      "10/10 [==============================] - 0s 682us/sample - loss: 3.2627\n",
      "Epoch 49/100\n",
      "10/10 [==============================] - 0s 576us/sample - loss: 3.1792\n",
      "Epoch 50/100\n",
      "10/10 [==============================] - 0s 639us/sample - loss: 3.1146\n",
      "Epoch 51/100\n",
      "10/10 [==============================] - 0s 643us/sample - loss: 3.0553\n",
      "Epoch 52/100\n",
      "10/10 [==============================] - 0s 555us/sample - loss: 2.9791\n",
      "Epoch 53/100\n",
      "10/10 [==============================] - 0s 707us/sample - loss: 2.9126\n",
      "Epoch 54/100\n",
      "10/10 [==============================] - 0s 472us/sample - loss: 2.8335\n",
      "Epoch 55/100\n",
      "10/10 [==============================] - 0s 713us/sample - loss: 2.7584\n",
      "Epoch 56/100\n",
      "10/10 [==============================] - 0s 531us/sample - loss: 2.6946\n",
      "Epoch 57/100\n",
      "10/10 [==============================] - 0s 641us/sample - loss: 2.6372\n",
      "Epoch 58/100\n",
      "10/10 [==============================] - 0s 744us/sample - loss: 2.5652\n",
      "Epoch 59/100\n",
      "10/10 [==============================] - 0s 758us/sample - loss: 2.5106\n",
      "Epoch 60/100\n",
      "10/10 [==============================] - 0s 453us/sample - loss: 2.4495\n",
      "Epoch 61/100\n",
      "10/10 [==============================] - 0s 863us/sample - loss: 2.3874\n",
      "Epoch 62/100\n",
      "10/10 [==============================] - 0s 753us/sample - loss: 2.3349\n",
      "Epoch 63/100\n",
      "10/10 [==============================] - 0s 748us/sample - loss: 2.2776\n",
      "Epoch 64/100\n",
      "10/10 [==============================] - 0s 709us/sample - loss: 2.2232\n",
      "Epoch 65/100\n",
      "10/10 [==============================] - 0s 811us/sample - loss: 2.1743\n",
      "Epoch 66/100\n",
      "10/10 [==============================] - 0s 657us/sample - loss: 2.1162\n",
      "Epoch 67/100\n",
      "10/10 [==============================] - 0s 848us/sample - loss: 2.0687\n",
      "Epoch 68/100\n",
      "10/10 [==============================] - 0s 853us/sample - loss: 2.0217\n",
      "Epoch 69/100\n",
      "10/10 [==============================] - 0s 660us/sample - loss: 1.9758\n",
      "Epoch 70/100\n",
      "10/10 [==============================] - 0s 677us/sample - loss: 1.9242\n",
      "Epoch 71/100\n",
      "10/10 [==============================] - 0s 721us/sample - loss: 1.8762\n",
      "Epoch 72/100\n",
      "10/10 [==============================] - 0s 539us/sample - loss: 1.8358\n",
      "Epoch 73/100\n",
      "10/10 [==============================] - 0s 674us/sample - loss: 1.7859\n",
      "Epoch 74/100\n",
      "10/10 [==============================] - 0s 648us/sample - loss: 1.7444\n",
      "Epoch 75/100\n",
      "10/10 [==============================] - 0s 487us/sample - loss: 1.7097\n",
      "Epoch 76/100\n",
      "10/10 [==============================] - 0s 723us/sample - loss: 1.6682\n",
      "Epoch 77/100\n",
      "10/10 [==============================] - 0s 602us/sample - loss: 1.6322\n",
      "Epoch 78/100\n",
      "10/10 [==============================] - 0s 536us/sample - loss: 1.5957\n",
      "Epoch 79/100\n",
      "10/10 [==============================] - 0s 581us/sample - loss: 1.5599\n",
      "Epoch 80/100\n",
      "10/10 [==============================] - 0s 620us/sample - loss: 1.5286\n",
      "Epoch 81/100\n",
      "10/10 [==============================] - 0s 608us/sample - loss: 1.4964\n",
      "Epoch 82/100\n",
      "10/10 [==============================] - 0s 396us/sample - loss: 1.4656\n",
      "Epoch 83/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.4331\n",
      "Epoch 84/100\n",
      "10/10 [==============================] - 0s 536us/sample - loss: 1.4004\n",
      "Epoch 85/100\n",
      "10/10 [==============================] - 0s 693us/sample - loss: 1.3685\n",
      "Epoch 86/100\n",
      "10/10 [==============================] - 0s 737us/sample - loss: 1.3348\n",
      "Epoch 87/100\n",
      "10/10 [==============================] - 0s 624us/sample - loss: 1.2977\n",
      "Epoch 88/100\n",
      "10/10 [==============================] - 0s 525us/sample - loss: 1.2673\n",
      "Epoch 89/100\n",
      "10/10 [==============================] - 0s 603us/sample - loss: 1.2327\n",
      "Epoch 90/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10/10 [==============================] - 0s 664us/sample - loss: 1.2015\n",
      "Epoch 91/100\n",
      "10/10 [==============================] - 0s 640us/sample - loss: 1.1767\n",
      "Epoch 92/100\n",
      "10/10 [==============================] - 0s 663us/sample - loss: 1.1513\n",
      "Epoch 93/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.1273\n",
      "Epoch 94/100\n",
      "10/10 [==============================] - 0s 436us/sample - loss: 1.0989\n",
      "Epoch 95/100\n",
      "10/10 [==============================] - 0s 691us/sample - loss: 1.0749\n",
      "Epoch 96/100\n",
      "10/10 [==============================] - 0s 551us/sample - loss: 1.0450\n",
      "Epoch 97/100\n",
      "10/10 [==============================] - 0s 656us/sample - loss: 1.0208\n",
      "Epoch 98/100\n",
      "10/10 [==============================] - 0s 667us/sample - loss: 0.9956\n",
      "Epoch 99/100\n",
      "10/10 [==============================] - 0s 701us/sample - loss: 0.9768\n",
      "Epoch 100/100\n",
      "10/10 [==============================] - 0s 703us/sample - loss: 0.9585\n",
      "k (Linear Algebra Method):\n",
      "[[2.10930333]\n",
      " [3.02422895]]\n",
      "Ground Truth, Linear Alg Prediction, MLP Prediction\n",
      "[[1.         1.11592945 1.92516315]\n",
      " [1.4        1.45079525 2.00976539]\n",
      " [1.8        1.58053541 2.04254341]\n",
      " [2.2        2.12543548 2.18020988]\n",
      " [2.6        2.71024564 2.3279593 ]\n",
      " [3.         3.08924675 2.42371202]\n",
      " [3.4        3.34323007 2.48787975]\n",
      " [3.8        3.82231032 2.60891676]\n",
      " [4.2        4.18731641 2.70113373]\n",
      " [4.6        4.57495521 2.79906869]]\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "# numpy package\n",
    "import numpy as np\n",
    "\n",
    "# keras modules\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.utils import plot_model\n",
    "\n",
    "# generate x data\n",
    "x = np.arange(-1,1,0.2)\n",
    "x = np.reshape(x, [-1,1])\n",
    "\n",
    "# generate y data\n",
    "y = 2 * x + 3\n",
    "\n",
    "# True if noise is added to y\n",
    "is_noisy = True\n",
    "\n",
    "# add noise if enabled\n",
    "if is_noisy:\n",
    "    noise = np.random.uniform(-0.1, 0.1, x.shape)\n",
    "    x = x + noise\n",
    "\n",
    "# deep learning method\n",
    "# build 2-layer MLP network \n",
    "model = Sequential()\n",
    "# 1st MLP has 8 units (perceptron), input is 1-dim\n",
    "model.add(Dense(units=8, input_dim=1))\n",
    "# 2nd MLP has 1 unit, output is 1-dim\n",
    "model.add(Dense(units=1))\n",
    "# print summary to double check the network\n",
    "model.summary()\n",
    "# create a nice image of the network model\n",
    "plot_model(model, to_file='linear-model.png', show_shapes=True)\n",
    "# indicate the loss function and use stochastic gradient descent\n",
    "# (sgd) as optimizer\n",
    "model.compile(loss='mse', optimizer='sgd')\n",
    "# feed the network with complete dataset (1 epoch) 100 times\n",
    "# batch size of sgd is 4\n",
    "model.fit(x, y, epochs=100, batch_size=4)\n",
    "# simple validation by predicting the output based on x\n",
    "ypred = model.predict(x)\n",
    "\n",
    "# linear algebra method\n",
    "ones = np.ones(x.shape)\n",
    "# A is the concat of x and 1s\n",
    "A = np.concatenate([x,ones], axis=1)\n",
    "# compute k using using pseudo-inverse\n",
    "k = np.matmul(np.linalg.pinv(A), y) \n",
    "print(\"k (Linear Algebra Method):\")\n",
    "print(k)\n",
    "# predict the output using linear algebra solution\n",
    "yla = np.matmul(A, k)\n",
    "\n",
    "# print ground truth, linear algebra, MLP solutions\n",
    "outputs = np.concatenate([y, yla, ypred], axis=1)\n",
    "print(\"Ground Truth, Linear Alg Prediction, MLP Prediction\")\n",
    "print(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
